package yatan.common.pageretriever;

import java.io.IOException;
import java.io.InputStream;
import java.util.zip.GZIPInputStream;

import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.entity.HttpEntityWrapper;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;

import yatan.common.proxypool.DelayProxyPool;
import yatan.common.proxypool.Proxy;
import yatan.common.utility.EncodeUtility;
import yatan.common.utility.ParamUtility;

public class PageRetriever {
    private static final Logger LOGGER = Logger.getLogger(PageRetriever.class);

    private boolean useProxy = true;
    private int retryLimit = 10;
    private int connectionTimeout = 5;
    private int socketTimeOut = 30;

    private PageValidator pageValidator;

    private HttpResponse lastResponse;
    
    public PageRetriever() {
    }

    public String getPageContent(String url) throws PageRetrieverException {
        ParamUtility.checkStringNullEmpty(url, "url");

        // create http client instance
        DefaultHttpClient httpClient = new DefaultHttpClient();

        httpClient.addRequestInterceptor(new HttpRequestInterceptor() {
            public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
        	request.setHeader("Accept", "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5");
                request
                        .setHeader(
                                "User-Agent",
                                "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.237 Safari/534.10");
                if (!request.containsHeader("Accept-Encoding")) {
                    request.addHeader("Accept-Encoding", "gzip");
                    
                }
            }

        });

        httpClient.addResponseInterceptor(new HttpResponseInterceptor() {
            public void process(final HttpResponse response, final HttpContext context) throws HttpException,
                    IOException {
                HttpEntity entity = response.getEntity();
                Header ceheader = entity.getContentEncoding();
                if (ceheader != null) {
                    HeaderElement[] codecs = ceheader.getElements();
                    for (int i = 0; i < codecs.length; i++) {
                        if (codecs[i].getName().equalsIgnoreCase("gzip")) {
                            response.setEntity(new GzipDecompressingEntity(response.getEntity()));
                            return;
                        }
                    }
                }
            }
        });

        // setup connection timeout
        HttpParams params = httpClient.getParams();
        HttpConnectionParams.setConnectionTimeout(params, this.connectionTimeout * 1000);
        HttpConnectionParams.setSoTimeout(params, this.socketTimeOut * 1000);
        httpClient.setParams(params);

        // try to get the url until retry time limit
        try {
            int retryTimes = 0;
            while (true) {
                String content = requestPageContent(httpClient, url, this.useProxy);
                // if content is not null, means url successfully requested
                if (content != null) {
                    return EncodeUtility.convertEncodedCharacters(content);
                }

                if (this.retryLimit > 0 && retryTimes++ >= this.retryLimit - 1) {
                    throw new PageRetrieverException("Reach retry limit " + this.getRetryLimit() + ", abort.");
                }
            }
        } finally {
            // shut down connection manager
            httpClient.getConnectionManager().shutdown();
        }
    }

    public void setPageValidator(PageValidator pageValidator) {
        this.pageValidator = pageValidator;
    }

    public PageValidator getPageValidator() {
        return pageValidator;
    }

    public void setRetryLimit(int retryLimit) {
        this.retryLimit = retryLimit;
    }

    public int getRetryLimit() {
        return retryLimit;
    }

    public void setSocketTimeOut(int connectionTimeOut) {
        this.socketTimeOut = connectionTimeOut;
    }

    public int getSocketTimeOut() {
        return socketTimeOut;
    }

    private String requestPageContent(DefaultHttpClient httpClient, String url, boolean useProxy) {
        // create get request object
        HttpGet get = new HttpGet(url);

        // get proxy from pool if needed
        Proxy proxy = null;
        if (useProxy) {
            proxy = DelayProxyPool.getInstance().nextProxy();
        }

        HttpResponse response = null;
        try {
            // setup proxy if proxy is not null
            if (proxy != null) {
                HttpHost proxyHost = new HttpHost(proxy.getHost(), proxy.getPort());
                httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost);
                LOGGER.debug("Using proxy " + proxy.getHost() + ":" + proxy.getPort());
            }

            // execute http get request
            LOGGER.debug("Sending get to " + url);
            this.lastResponse = response = httpClient.execute(get);
            if (response.getStatusLine().getStatusCode() == 200) {
                // if response code is 200, get the entity and validate
                String content = EntityUtils.toString(response.getEntity());
                if (this.pageValidator == null || this.pageValidator.isPageGood(url, content)) {
                    // validate success, report proxy OK if necessary
                    if (proxy != null) {
                        DelayProxyPool.getInstance().reportProxyOK(proxy);
                    }

                    // return page content
                    return content;
                }
            }

            // at this point the request is already failed due to some reason
            // consume the http response entity to release resouces
            if (response.getEntity() != null) {
                response.getEntity().consumeContent();
            }
        } catch (ClientProtocolException e) {
            get.abort();
        } catch (IllegalStateException e) {
            get.abort();
        } catch (IOException e) {
            get.abort();
        } finally {
            if (response != null && response.getEntity() != null) {
                try {
                    response.getEntity().consumeContent();
                } catch (IOException e) {
                    // ignore
                }
            }
        }

        // report proxy fail if proxy is not null
        if (proxy != null) {
            DelayProxyPool.getInstance().reportProxyFail(proxy);
        }
        return null;
    }

    public void setUseProxy(boolean useProxy) {
        this.useProxy = useProxy;
    }

    public boolean isUseProxy() {
        return useProxy;
    }

    public void setConnectionTimeout(int connectionTimeout) {
        this.connectionTimeout = connectionTimeout;
    }

    public int getConnectionTimeout() {
        return connectionTimeout;
    }

    public HttpResponse getLastResponse() {
	return lastResponse;
    }

    private static class GzipDecompressingEntity extends HttpEntityWrapper {
        public GzipDecompressingEntity(final HttpEntity entity) {
            super(entity);
        }

        @Override
        public InputStream getContent() throws IOException, IllegalStateException {

            // the wrapped entity's getContent() decides about repeatability
            InputStream wrappedin = wrappedEntity.getContent();

            return new GZIPInputStream(wrappedin);
        }

        @Override
        public long getContentLength() {
            // length of ungzipped content is not known
            return -1;
        }
    }
}
